package DOM;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.io.FileUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.xml.sax.SAXException;

public class ArticlesFilter {
	
	/**********************************************/
	public static String filterArticle(
		String url, String RegTitle, String RegDate, String RegAuth, String RegCateg, String RegContent){

		Document doc = null;
		String result = "<?xml version=\"1.0\"?>\n";
		 
		// need http protocol
		try {
			doc = Jsoup.parse(new File(url), "UTF-8");
		} catch (IOException e) {
			e.printStackTrace();
		}
 
		// get title
		Elements titlelinks = doc.select(RegTitle);
		String title = titlelinks.text();
		
		// get date
		String date = "";
		Elements metalinks = doc.select(RegDate);
		for (Element element : metalinks) {
			Date tmp;
			if((tmp = ArticleAnalyze.dateAnalyzer(element.toString()))!=null){
				date += tmp;
			}
		}
		
		// get author
		String auth = "";
		Elements authlinks = doc.select(RegAuth);
		for (Element element : authlinks) {
			auth += element.text();
		}
		
		// get category
		String cate = "";
		Elements catelinks = doc.select(RegCateg);
		for (Element element : catelinks) {
			cate += element.text()+", ";
		}
		if(!cate.isEmpty()){
			cate = cate.trim().substring(0, cate.trim().length()-1);
		}
		
		// get content
		String content = "";
		Elements contentlinks = doc.select(RegContent);
		for (Element link : contentlinks) {
			content += link.text();
		}
		
		//filter category
		cate = cate.replaceAll("​", "");
		String tmpCate = "";
		if(cate.matches(".*[,|/].*")){
			for (String s : cate.split("[,|/]")) {
				if(Constant.categoryDic().containsKey(s.trim()) 
						&& !tmpCate.matches(".*"+Constant.categoryDic().get(s.trim())+".*"))
					tmpCate += Constant.categoryDic().get(s.trim())+", ";
			}
			if(tmpCate.length() > 1)
				tmpCate = tmpCate.substring(0, tmpCate.length()-2);
		}else{
			if(Constant.categoryDic().containsKey(cate.trim()))
				tmpCate = Constant.categoryDic().get(cate.trim());
			else
				System.out.println(cate+" no contain");
		}
		
		if (tmpCate.isEmpty()){
			Constant.numNullCategory++;
			return null;
		}else{
			if(Constant.dateDictionry().containsKey(tmpCate))
				System.out.println(tmpCate+": "+url);
		}
		
		content = Utils.filterContent(content);
		
		if(content.matches("[a-zA-z0-0]"))
			System.out.println("Contain English");
		
		result +="<article>\n";
		result += "\t<category>"+tmpCate+"</category>\n";
		result += "\t<title>"+title+"</title>\n";
		result += "\t<date>"+date+"</date>\n";
		result += "\t<author>"+Utils.AuthorFilter(auth)+"</author>\n";
		result += "\t<content>"+content+"</content>\n";
		result += "</article>";
		
		if(date.isEmpty()) Constant.numNullDate++;
		if(auth.isEmpty()) Constant.numNullAuth++;		
		if(cate.trim().equals("English") || content.isEmpty()){
			if(content.isEmpty()) Constant.numNullContent++;
			return null;
		}
		
		return result;
	}
	
	public static void filterArticles(
			String dir, String RegTitle, String RegDate, String RegAuth, String RegCateg, String RegContent){
			
		File dirs = new File(dir);
		ArrayList<File> fileA = new ArrayList<File>();
		if(dirs.isDirectory()){
			File dirXML = new File(dir+"/../xml/xml-"+dirs.getName());
			if(!dirXML.mkdirs()) 
				System.out.println("Directory XML can't be created");
			
			Utils.flateFile(dirs, fileA);
			
			int i = 1;
			for (File file : fileA) {
				if(!file.isDirectory()){
					try {
						File outf = new File(dirXML.getAbsolutePath()+"/"+file.getName());
						if(!outf.exists()){
							i = 1;
							String tmp = filterArticle(file.getAbsolutePath(), RegTitle, RegDate, RegAuth, RegCateg, RegContent);
							if(tmp!=null){
								FileOutputStream out = new FileOutputStream(outf);
								out.write(tmp.getBytes());
								out.flush();
								out.close();
							}
						}else{
							String tmp = filterArticle(file.getAbsolutePath(), RegTitle, RegDate, RegAuth, RegCateg, RegContent);
							if(tmp!=null){
								FileOutputStream out = new FileOutputStream(
										new File(dirXML.getAbsolutePath()+"/"+file.getName()+" ("+i+")"));
								out.write(tmp.getBytes());
								out.flush();
								out.close();
								
								i++;
							}
						}
					} catch (FileNotFoundException e) {
						e.printStackTrace();
					} catch (IOException e) {
						e.printStackTrace();
					}
				}
			}
			
			
		}else{
			System.out.println(dir+" is not directory.");
		}
	}
	
	/**********************************************/
	public static void categoryGrouping(String path){
		int numPhrase = 0, numWords = 0;
		
		ArrayList<File> fileA = new ArrayList<File>();
		Utils.flateFile(new File(path), fileA);
		
		Document doc = null;
		
		// need http protocol
		try {
			for (File f : fileA) {
				doc = Jsoup.parse(f, "UTF-8");
				
				//------------------------------------
				numPhrase += Utils.countPhrase(doc.select("content").text());
				numWords += Utils.countWords(doc.select("content").text());
				
				//------------------------------------
				String tmpp = doc.select("category").text();
//				System.out.println(tmpp);
				String[] tmp = tmpp.split(",");
				for (String s : tmp) {
					s = s.trim();
					if(!Constant.categoryDic().containsKey(s)){
						System.out.println(s+" category doesn't contained.");
					}
					
					File dirL1 = new File(path+"/../category - "+path.split("/")[path.split("/").length-1]);
					if(!dirL1.exists())
						dirL1.mkdirs();
					
					File dirL2 = new File(dirL1+"/"+s);
					if(!dirL2.exists())
						dirL2.mkdirs();
					
					File des = new File(dirL2+"/"+f.getName());
					if(!des.exists())
						FileUtils.copyFile(f, des);
					
				}
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		System.out.println("**************");
		
		System.out.println("Number of file: "+fileA.size());
		System.out.println("Number of phrases: "+numPhrase);
		System.out.println("Number of words: "+numWords);
		
		System.out.println("****************");
		System.out.println("Error:");
		System.out.println("Miss category: "+Constant.numNullCategory);
		System.out.println("Miss content: "+Constant.numNullContent);
		System.out.println("Miss author: "+Constant.numNullAuth);
		System.out.println("Miss date: "+Constant.numNullDate);
	}
	
	public static void groupCateContent(String path, String contentOuput){
		int numPhrase = 0, numWords = 0;
		ArrayList<String> listCategory = new ArrayList<String>();
		
		File catFile = new File(path);
		
		ArrayList<File> fileA = new ArrayList<File>();
		Utils.flateFile(catFile, fileA);
		FileOutputStream in;
		
		Document doc = null;
		
		// need http protocol
		try {
			in = new FileOutputStream(new File(contentOuput+"/"+catFile.getName()+"-input.txt"));
			for (File f : fileA) {
				doc = Jsoup.parse(f, "UTF-8");
				
				in.write(doc.select("content").text().getBytes());
				
				//------------------------------------
				numPhrase += Utils.countPhrase(doc.select("content").text());
				numWords += Utils.countWords(doc.select("content").text());
				
				//------------------------------------
				String[] tmp = doc.select("category").text().split(",");
				for (String s : tmp) {
					s = s.trim();
					if(!listCategory.contains(s)){
//						System.out.println(s);
						listCategory.add(s);
					}					
				}
								
			}
			in.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		//--------------------------------
//		for (String str : listCategory) {
//			System.out.println("cate = "+str);
//		}
		System.out.println("cate: "+catFile.getName());
		
		System.out.println("Number of file: "+fileA.size());
		System.out.println("Number of phrases: "+numPhrase);
		System.out.println("Number of words: "+numWords);
	}
	
	public static void viewCategoriesInfo(String path){
		File dir = new File(path);
		for (File f : dir.listFiles()) {
			System.out.println("**********************");
			System.out.println(f.getName());
			groupCateContent(f.getAbsolutePath(), "D:/drop box/Dropbox/i5/semester 2/intern at school/inputs");
		}
	}

}
